{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "24ae5547",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = '0'\n",
    "os.environ['TF_FORCE_GPU_ALLOW_GROWTH'] = 'true'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "3395870c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/.local/lib/python3.8/site-packages/apex/pyprof/__init__.py:5: FutureWarning: pyprof will be removed by the end of June, 2022\n",
      "  warnings.warn(\"pyprof will be removed by the end of June, 2022\", FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "from transformers import T5Config, TFT5Model, T5Model, load_tf_weights_in_t5, T5Tokenizer\n",
    "from transformers import T5ForConditionalGeneration, TFT5ForConditionalGeneration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "06db4241",
   "metadata": {},
   "outputs": [],
   "source": [
    "!rm -rf temp2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a541b0ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "out = 'temp2'\n",
    "os.makedirs(out, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "48753c21",
   "metadata": {},
   "outputs": [],
   "source": [
    "config = T5Config.from_pretrained('malay-huggingface/t5-tiny-bahasa-cased')\n",
    "config.save_pretrained(out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "50a47b1d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "checkpoint\r\n",
      "events.out.tfevents.1659110853.huseincomel-desktop\r\n",
      "graph.pbtxt\r\n",
      "model.ckpt-1024000.data-00000-of-00002\r\n",
      "model.ckpt-1024000.data-00001-of-00002\r\n",
      "model.ckpt-1024000.index\r\n",
      "model.ckpt-1024000.meta\r\n",
      "model.ckpt-1048000.data-00000-of-00002\r\n",
      "model.ckpt-1048000.data-00001-of-00002\r\n",
      "model.ckpt-1048000.index\r\n",
      "model.ckpt-1048000.meta\r\n",
      "model.ckpt-1072000.data-00000-of-00002\r\n",
      "model.ckpt-1072000.data-00001-of-00002\r\n",
      "model.ckpt-1072000.index\r\n",
      "model.ckpt-1072000.meta\r\n",
      "model.ckpt-1095000.data-00000-of-00002\r\n",
      "model.ckpt-1095000.data-00001-of-00002\r\n",
      "model.ckpt-1095000.index\r\n",
      "model.ckpt-1095000.meta\r\n",
      "model.ckpt-1119000.data-00000-of-00002\r\n",
      "model.ckpt-1119000.data-00001-of-00002\r\n",
      "model.ckpt-1119000.index\r\n",
      "model.ckpt-1119000.meta\r\n",
      "model.ckpt-1143000.data-00000-of-00002\r\n",
      "model.ckpt-1143000.data-00001-of-00002\r\n",
      "model.ckpt-1143000.index\r\n",
      "model.ckpt-1143000.meta\r\n",
      "model.ckpt-1166000.data-00000-of-00002\r\n",
      "model.ckpt-1166000.data-00001-of-00002\r\n",
      "model.ckpt-1166000.index\r\n",
      "model.ckpt-1166000.meta\r\n",
      "model.ckpt-1190000.data-00000-of-00002\r\n",
      "model.ckpt-1190000.data-00001-of-00002\r\n",
      "model.ckpt-1190000.index\r\n",
      "model.ckpt-1190000.meta\r\n",
      "model.ckpt-1214000.data-00000-of-00002\r\n",
      "model.ckpt-1214000.data-00001-of-00002\r\n",
      "model.ckpt-1214000.index\r\n",
      "model.ckpt-1214000.meta\r\n",
      "model.ckpt-1237000.data-00000-of-00002\r\n",
      "model.ckpt-1237000.data-00001-of-00002\r\n",
      "model.ckpt-1237000.index\r\n",
      "model.ckpt-1237000.meta\r\n",
      "model.ckpt-1261000.data-00000-of-00002\r\n",
      "model.ckpt-1261000.data-00001-of-00002\r\n",
      "model.ckpt-1261000.index\r\n",
      "model.ckpt-1261000.meta\r\n",
      "model.ckpt-1283000.data-00000-of-00002\r\n",
      "model.ckpt-1283000.data-00001-of-00002\r\n",
      "model.ckpt-1283000.index\r\n",
      "model.ckpt-1283000.meta\r\n",
      "model.ckpt-1298000.data-00000-of-00002\r\n",
      "model.ckpt-1298000.data-00001-of-00002\r\n",
      "model.ckpt-1298000.index\r\n",
      "model.ckpt-1298000.meta\r\n",
      "model.ckpt-1299000.data-00000-of-00002\r\n",
      "model.ckpt-1299000.data-00001-of-00002\r\n",
      "model.ckpt-1299000.index\r\n",
      "model.ckpt-1299000.meta\r\n",
      "model.ckpt-1300000.data-00000-of-00002\r\n",
      "model.ckpt-1300000.data-00001-of-00002\r\n",
      "model.ckpt-1300000.index\r\n",
      "model.ckpt-1300000.meta\r\n",
      "model.ckpt-1301000.data-00000-of-00002\r\n",
      "model.ckpt-1301000.data-00001-of-00002\r\n",
      "model.ckpt-1301000.index\r\n",
      "model.ckpt-1301000.meta\r\n",
      "model.ckpt-1302000.data-00000-of-00002\r\n",
      "model.ckpt-1302000.data-00001-of-00002\r\n",
      "model.ckpt-1302000.index\r\n",
      "model.ckpt-1302000.meta\r\n",
      "operative_config.gin\r\n"
     ]
    }
   ],
   "source": [
    "!ls t5-tiny-noisy-en-ms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "f54394ad",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "T5Model(\n",
       "  (shared): Embedding(32128, 384)\n",
       "  (encoder): T5Stack(\n",
       "    (embed_tokens): Embedding(32128, 384)\n",
       "    (block): ModuleList(\n",
       "      (0): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=384, bias=False)\n",
       "              (relative_attention_bias): Embedding(32, 12)\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseActDense(\n",
       "              (wi): Linear(in_features=384, out_features=1344, bias=False)\n",
       "              (wo): Linear(in_features=1344, out_features=384, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "              (act): ReLU()\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (1): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=384, bias=False)\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseActDense(\n",
       "              (wi): Linear(in_features=384, out_features=1344, bias=False)\n",
       "              (wo): Linear(in_features=1344, out_features=384, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "              (act): ReLU()\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (2): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=384, bias=False)\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseActDense(\n",
       "              (wi): Linear(in_features=384, out_features=1344, bias=False)\n",
       "              (wo): Linear(in_features=1344, out_features=384, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "              (act): ReLU()\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (3): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=384, bias=False)\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseActDense(\n",
       "              (wi): Linear(in_features=384, out_features=1344, bias=False)\n",
       "              (wo): Linear(in_features=1344, out_features=384, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "              (act): ReLU()\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (final_layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "    (dropout): Dropout(p=0.1, inplace=False)\n",
       "  )\n",
       "  (decoder): T5Stack(\n",
       "    (embed_tokens): Embedding(32128, 384)\n",
       "    (block): ModuleList(\n",
       "      (0): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=384, bias=False)\n",
       "              (relative_attention_bias): Embedding(32, 12)\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerCrossAttention(\n",
       "            (EncDecAttention): T5Attention(\n",
       "              (q): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=384, bias=False)\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (2): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseActDense(\n",
       "              (wi): Linear(in_features=384, out_features=1344, bias=False)\n",
       "              (wo): Linear(in_features=1344, out_features=384, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "              (act): ReLU()\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (1): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=384, bias=False)\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerCrossAttention(\n",
       "            (EncDecAttention): T5Attention(\n",
       "              (q): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=384, bias=False)\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (2): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseActDense(\n",
       "              (wi): Linear(in_features=384, out_features=1344, bias=False)\n",
       "              (wo): Linear(in_features=1344, out_features=384, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "              (act): ReLU()\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (2): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=384, bias=False)\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerCrossAttention(\n",
       "            (EncDecAttention): T5Attention(\n",
       "              (q): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=384, bias=False)\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (2): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseActDense(\n",
       "              (wi): Linear(in_features=384, out_features=1344, bias=False)\n",
       "              (wo): Linear(in_features=1344, out_features=384, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "              (act): ReLU()\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (3): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=384, bias=False)\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerCrossAttention(\n",
       "            (EncDecAttention): T5Attention(\n",
       "              (q): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=384, bias=False)\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (2): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseActDense(\n",
       "              (wi): Linear(in_features=384, out_features=1344, bias=False)\n",
       "              (wo): Linear(in_features=1344, out_features=384, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "              (act): ReLU()\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (final_layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "    (dropout): Dropout(p=0.1, inplace=False)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = T5Model(config)\n",
    "load_tf_weights_in_t5(model, config, 't5-tiny-noisy-en-ms/model.ckpt-1302000')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "86b9d748",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "tokenizer = T5Tokenizer('sp10m.cased.ms-en.model', padding = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "7e68dcc6",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save_pretrained(out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "737e7c80",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_gen = T5ForConditionalGeneration.from_pretrained(out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "005b412a",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-08-01 10:53:01.824014: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-08-01 10:53:01.827858: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-08-01 10:53:01.828616: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-08-01 10:53:01.829507: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 AVX512F FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2022-08-01 10:53:01.830183: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-08-01 10:53:01.830870: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-08-01 10:53:01.831492: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-08-01 10:53:06.056010: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-08-01 10:53:06.056640: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-08-01 10:53:06.057237: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-08-01 10:53:06.057821: W tensorflow/core/common_runtime/gpu/gpu_bfc_allocator.cc:39] Overriding allow_growth setting because the TF_FORCE_GPU_ALLOW_GROWTH environment variable is set. Original config value was 0.\n",
      "2022-08-01 10:53:06.057845: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1510] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 15228 MB memory:  -> device: 0, name: NVIDIA GeForce RTX 3090 Ti, pci bus id: 0000:01:00.0, compute capability: 8.6\n",
      "2022-08-01 10:53:06.069290: W tensorflow/python/util/util.cc:348] Sets are not currently considered sequences, but this may change in the future, so consider avoiding using them.\n",
      "2022-08-01 10:53:06.878599: I tensorflow/stream_executor/cuda/cuda_blas.cc:1760] TensorFloat-32 will be used for the matrix multiplication. This will only be logged once.\n",
      "Some weights of the PyTorch model were not used when initializing the TF 2.0 model TFT5ForConditionalGeneration: ['encoder.embed_tokens.weight', 'decoder.embed_tokens.weight']\n",
      "- This IS expected if you are initializing TFT5ForConditionalGeneration from a PyTorch model trained on another task or with another architecture (e.g. initializing a TFBertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing TFT5ForConditionalGeneration from a PyTorch model that you expect to be exactly identical (e.g. initializing a TFBertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "All the weights of TFT5ForConditionalGeneration were initialized from the PyTorch model.\n",
      "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFT5ForConditionalGeneration for predictions without further training.\n"
     ]
    }
   ],
   "source": [
    "model_tf = TFT5ForConditionalGeneration.from_pretrained(out, from_pt = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "fc1a66fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('en-ms-right.test') as fopen:\n",
    "    right = fopen.read().split('\\n')\n",
    "    \n",
    "with open('en-ms-left.test') as fopen:\n",
    "    left = fopen.read().split('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "3124e42d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "T5ForConditionalGeneration(\n",
       "  (shared): Embedding(32128, 384)\n",
       "  (encoder): T5Stack(\n",
       "    (embed_tokens): Embedding(32128, 384)\n",
       "    (block): ModuleList(\n",
       "      (0): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=384, bias=False)\n",
       "              (relative_attention_bias): Embedding(32, 12)\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseActDense(\n",
       "              (wi): Linear(in_features=384, out_features=1344, bias=False)\n",
       "              (wo): Linear(in_features=1344, out_features=384, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "              (act): ReLU()\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (1): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=384, bias=False)\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseActDense(\n",
       "              (wi): Linear(in_features=384, out_features=1344, bias=False)\n",
       "              (wo): Linear(in_features=1344, out_features=384, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "              (act): ReLU()\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (2): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=384, bias=False)\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseActDense(\n",
       "              (wi): Linear(in_features=384, out_features=1344, bias=False)\n",
       "              (wo): Linear(in_features=1344, out_features=384, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "              (act): ReLU()\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (3): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=384, bias=False)\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseActDense(\n",
       "              (wi): Linear(in_features=384, out_features=1344, bias=False)\n",
       "              (wo): Linear(in_features=1344, out_features=384, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "              (act): ReLU()\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (final_layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "    (dropout): Dropout(p=0.1, inplace=False)\n",
       "  )\n",
       "  (decoder): T5Stack(\n",
       "    (embed_tokens): Embedding(32128, 384)\n",
       "    (block): ModuleList(\n",
       "      (0): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=384, bias=False)\n",
       "              (relative_attention_bias): Embedding(32, 12)\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerCrossAttention(\n",
       "            (EncDecAttention): T5Attention(\n",
       "              (q): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=384, bias=False)\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (2): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseActDense(\n",
       "              (wi): Linear(in_features=384, out_features=1344, bias=False)\n",
       "              (wo): Linear(in_features=1344, out_features=384, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "              (act): ReLU()\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (1): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=384, bias=False)\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerCrossAttention(\n",
       "            (EncDecAttention): T5Attention(\n",
       "              (q): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=384, bias=False)\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (2): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseActDense(\n",
       "              (wi): Linear(in_features=384, out_features=1344, bias=False)\n",
       "              (wo): Linear(in_features=1344, out_features=384, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "              (act): ReLU()\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (2): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=384, bias=False)\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerCrossAttention(\n",
       "            (EncDecAttention): T5Attention(\n",
       "              (q): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=384, bias=False)\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (2): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseActDense(\n",
       "              (wi): Linear(in_features=384, out_features=1344, bias=False)\n",
       "              (wo): Linear(in_features=1344, out_features=384, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "              (act): ReLU()\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (3): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=384, bias=False)\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerCrossAttention(\n",
       "            (EncDecAttention): T5Attention(\n",
       "              (q): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=384, bias=False)\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (2): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseActDense(\n",
       "              (wi): Linear(in_features=384, out_features=1344, bias=False)\n",
       "              (wo): Linear(in_features=1344, out_features=384, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "              (act): ReLU()\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (final_layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "    (dropout): Dropout(p=0.1, inplace=False)\n",
       "  )\n",
       "  (lm_head): Linear(in_features=384, out_features=32128, bias=False)\n",
       ")"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_gen.cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "35abd80e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████| 69744/69744 [3:49:10<00:00,  5.07it/s]\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "batch_size = 1\n",
    "\n",
    "results = []\n",
    "for i in tqdm(range(0, len(left), batch_size)):\n",
    "    input_ids = [{'input_ids': tokenizer.encode(f'terjemah Inggeris ke Melayu: {s}', return_tensors = 'pt')[0]} for s in left[i:i + batch_size]]\n",
    "    padded = tokenizer.pad(input_ids, padding = 'longest')\n",
    "    outputs = model_gen.generate(padded['input_ids'].cuda(), attention_mask = padded['attention_mask'].cuda(), max_length = 1000)\n",
    "    for o in outputs:\n",
    "        results.append(tokenizer.decode(o, skip_special_tokens=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "9a28a3e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# from tqdm import tqdm\n",
    "\n",
    "# batch_size = 2\n",
    "\n",
    "# results = []\n",
    "# for i in tqdm(range(0, len(left[:1]), batch_size)):\n",
    "#     input_ids = [{'input_ids': tokenizer.encode(f'terjemah Inggeris ke Melayu: {s}', return_tensors = 'pt')[0]} for s in left[i:i + batch_size]]\n",
    "#     padded = tokenizer.pad(input_ids, padding = 'longest')\n",
    "#     outputs = model_gen.generate(**padded, max_length = 1000)\n",
    "#     for o in outputs:\n",
    "#         results.append(tokenizer.decode(o, skip_special_tokens=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "52c3b550",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Anda tahu bagaimana kisah itu berlaku. Dua orang yang heteroseksual, hampir pasti orang kulit putih berjuang dalam hidup mereka. Mereka bertemu satu sama lain, berkumpul, berpisah, berkumpul semula dan mencari kebahagiaan dan penyelesaian yang benar di antara satu sama lain. Akhirnya, mereka dapat memulakan kehidupan normal! Rom-com dapat menjadi pelarian yang menyenangkan, tetapi kisah yang diceritakan terlalu sering, yang terlalu sering tidak mempunyai kepelbagaian, terlalu bergantung pada stereotaip jantina dan terlalu peduli dengan menjual jenama cinta yang mustahil untuk hidup: kajian tahun 2008 di Universiti Heriot Watt mendapati bahawa rom-com mempunyai kesan negatif terhadap hubungan, membuat kita mengejar standard cinta yang tidak dapat dicapai. Dalam proses menulis drama baru saya, Ross & Rachel, yang menghadapi mitos cinta moden dan dibuka di Edinburgh Festival Fringe pada bulan Ogos ini, saya harus banyak memikirkan percintaan dalam fiksyen dari Romeo dan Juliet hingga Pride dan Prejudice sesuai untuk Notting Hill. Mengapa kita terus terus menceritakan kisah yang sama?'"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "eb550d6f",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sacrebleu.metrics import BLEU, CHRF, TER\n",
    "\n",
    "bleu = BLEU()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "20530f31",
   "metadata": {},
   "outputs": [],
   "source": [
    "filtered_left, filtered_right = [], []\n",
    "for no, r in enumerate(results):\n",
    "    if len(r):\n",
    "        filtered_left.append(r)\n",
    "        filtered_right.append(right[no])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "e50b3fbd",
   "metadata": {},
   "outputs": [],
   "source": [
    "refs = [filtered_right]\n",
    "sys = filtered_left"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "361cc363",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Anda tahu bagaimana kisah itu berlaku. Dua orang yang heteroseksual, hampir pasti orang kulit putih berjuang dalam hidup mereka. Mereka bertemu satu sama lain, berkumpul, berpisah, berkumpul semula dan mencari kebahagiaan dan penyelesaian yang benar di antara satu sama lain. Akhirnya, mereka dapat memulakan kehidupan normal! Rom-com dapat menjadi pelarian yang menyenangkan, tetapi kisah yang diceritakan terlalu sering, yang terlalu sering tidak mempunyai kepelbagaian, terlalu bergantung pada stereotaip jantina dan terlalu peduli dengan menjual jenama cinta yang mustahil untuk hidup: kajian tahun 2008 di Universiti Heriot Watt mendapati bahawa rom-com mempunyai kesan negatif terhadap hubungan, membuat kita mengejar standard cinta yang tidak dapat dicapai. Dalam proses menulis drama baru saya, Ross & Rachel, yang menghadapi mitos cinta moden dan dibuka di Edinburgh Festival Fringe pada bulan Ogos ini, saya harus banyak memikirkan percintaan dalam fiksyen dari Romeo dan Juliet hingga Pride dan Prejudice sesuai untuk Notting Hill. Mengapa kita terus terus menceritakan kisah yang sama?'"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sys[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "2ba0ce1c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# for i in range(len(filtered_left)):\n",
    "#     print(filtered_left[i])\n",
    "#     print(filtered_right[i])\n",
    "#     print(right[i])\n",
    "#     print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "afa050d8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# filtered_right"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "ecc03af7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# filtered_left"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "e5d6f81d",
   "metadata": {},
   "outputs": [],
   "source": [
    "r = bleu.corpus_score(sys, refs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "cf03103f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'name': 'BLEU',\n",
       " 'score': 62.34308405954152,\n",
       " '_mean': -1.0,\n",
       " '_ci': -1.0,\n",
       " '_verbose': '82.6/67.5/57.2/49.3 (BP = 0.990 ratio = 0.990 hyp_len = 2604652 ref_len = 2630014)',\n",
       " 'bp': 0.9903100596731002,\n",
       " 'counts': [2152274, 1710097, 1409669, 1180379],\n",
       " 'totals': [2604652, 2534909, 2465191, 2395664],\n",
       " 'sys_len': 2604652,\n",
       " 'ref_len': 2630014,\n",
       " 'precisions': [82.63192165402518,\n",
       "  67.46186943988917,\n",
       "  57.18295255823991,\n",
       "  49.271475465674655],\n",
       " 'prec_str': '82.6/67.5/57.2/49.3',\n",
       " 'ratio': 0.990356705325523}"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r.__dict__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "88704b13",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer.push_to_hub('t5-tiny-finetuned-noisy-en-ms', organization='mesolitica')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "97afc0d5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2262632d219e4d19b65a1d34632d3e00",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Upload file pytorch_model.bin:   0%|          | 4.00k/133M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "To https://huggingface.co/mesolitica/t5-tiny-finetuned-noisy-en-ms\n",
      "   b05520d..9d4ca97  main -> main\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'https://huggingface.co/mesolitica/t5-tiny-finetuned-noisy-en-ms/commit/9d4ca97c676624111999d4984253c994eda3f617'"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_gen.push_to_hub('t5-tiny-finetuned-noisy-en-ms', organization='mesolitica')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "db817c07",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "97d22d5354374239a6221168ae8a2ab5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Upload file tf_model.h5:   0%|          | 4.00k/133M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "To https://huggingface.co/mesolitica/t5-tiny-finetuned-noisy-en-ms\n",
      "   9d4ca97..3e28c7a  main -> main\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'https://huggingface.co/mesolitica/t5-tiny-finetuned-noisy-en-ms/commit/3e28c7a2e41e8c4a8020065156ca534347ec723e'"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_tf.push_to_hub('t5-tiny-finetuned-noisy-en-ms', organization='mesolitica')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "2819f991",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Already up to date.\r\n"
     ]
    }
   ],
   "source": [
    "!cd t5-tiny-finetuned-noisy-en-ms && git pull"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "df9b8117",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[main be0d9b5] add tensorboard\n",
      " 1 file changed, 2 insertions(+), 2 deletions(-)\n",
      "Uploading LFS objects: 100% (1/1), 61 MB | 6.4 MB/s, done.                      \n",
      "Enumerating objects: 5, done.\n",
      "Counting objects: 100% (5/5), done.\n",
      "Delta compression using up to 16 threads\n",
      "Compressing objects: 100% (3/3), done.\n",
      "Writing objects: 100% (3/3), 374 bytes | 374.00 KiB/s, done.\n",
      "Total 3 (delta 1), reused 0 (delta 0)\n",
      "To https://huggingface.co/mesolitica/t5-tiny-finetuned-noisy-en-ms\n",
      "   3e28c7a..be0d9b5  main -> main\n"
     ]
    }
   ],
   "source": [
    "!cp t5-tiny-noisy-en-ms/*.tfevents.* t5-tiny-finetuned-noisy-en-ms\n",
    "!cd t5-tiny-finetuned-noisy-en-ms && git add . && git commit -m 'add tensorboard' && git push"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
